{
    "cells": [
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "<i>Copyright (c) Recommenders contributors.</i>\n",
                "\n",
                "<i>Licensed under the MIT License.</i>"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## FastAI Recommender\n",
                "\n",
                "This notebook shows how to use the [FastAI](https://fast.ai) recommender which is using [Pytorch](https://pytorch.org/) under the hood. "
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 1,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "System version: 3.9.16 (main, May 15 2023, 23:46:34) \n",
                        "[GCC 11.2.0]\n",
                        "Pandas version: 1.5.3\n",
                        "Fast AI version: 2.7.11\n",
                        "Torch version: 1.13.1+cu117\n",
                        "CUDA Available: True\n",
                        "CuDNN Enabled: True\n"
                    ]
                }
            ],
            "source": [
                "# Suppress all warnings\n",
                "import warnings\n",
                "warnings.filterwarnings(\"ignore\")\n",
                "\n",
                "import os\n",
                "import sys\n",
                "import numpy as np\n",
                "import pandas as pd\n",
                "import torch\n",
                "import fastai\n",
                "from tempfile import TemporaryDirectory\n",
                "\n",
                "from fastai.collab import collab_learner, CollabDataLoaders, load_learner\n",
                "\n",
                "from recommenders.utils.constants import (\n",
                "    DEFAULT_USER_COL as USER, \n",
                "    DEFAULT_ITEM_COL as ITEM, \n",
                "    DEFAULT_RATING_COL as RATING, \n",
                "    DEFAULT_TIMESTAMP_COL as TIMESTAMP, \n",
                "    DEFAULT_PREDICTION_COL as PREDICTION\n",
                ") \n",
                "from recommenders.utils.timer import Timer\n",
                "from recommenders.datasets import movielens\n",
                "from recommenders.datasets.python_splitters import python_stratified_split\n",
                "from recommenders.models.fastai.fastai_utils import cartesian_product, score\n",
                "from recommenders.evaluation.python_evaluation import map, ndcg_at_k, precision_at_k, recall_at_k\n",
                "from recommenders.evaluation.python_evaluation import rmse, mae, rsquared, exp_var\n",
                "from recommenders.utils.notebook_utils import store_metadata\n",
                "\n",
                "print(\"System version: {}\".format(sys.version))\n",
                "print(\"Pandas version: {}\".format(pd.__version__))\n",
                "print(\"Fast AI version: {}\".format(fastai.__version__))\n",
                "print(\"Torch version: {}\".format(torch.__version__))\n",
                "print(\"CUDA Available: {}\".format(torch.cuda.is_available()))\n",
                "print(\"CuDNN Enabled: {}\".format(torch.backends.cudnn.enabled))"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "Defining some constants to refer to the different columns of our dataset."
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 2,
            "metadata": {
                "tags": [
                    "parameters"
                ]
            },
            "outputs": [],
            "source": [
                "# top k items to recommend\n",
                "TOP_K = 10\n",
                "\n",
                "# Select MovieLens data size: 100k, 1m, 10m, or 20m\n",
                "MOVIELENS_DATA_SIZE = '100k'\n",
                "\n",
                "# Model parameters\n",
                "N_FACTORS = 40\n",
                "EPOCHS = 5"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 3,
            "metadata": {},
            "outputs": [
                {
                    "name": "stderr",
                    "output_type": "stream",
                    "text": [
                        "100%|██████████| 4.81k/4.81k [00:01<00:00, 3.52kKB/s]\n"
                    ]
                },
                {
                    "data": {
                        "text/html": [
                            "<div>\n",
                            "<style scoped>\n",
                            "    .dataframe tbody tr th:only-of-type {\n",
                            "        vertical-align: middle;\n",
                            "    }\n",
                            "\n",
                            "    .dataframe tbody tr th {\n",
                            "        vertical-align: top;\n",
                            "    }\n",
                            "\n",
                            "    .dataframe thead th {\n",
                            "        text-align: right;\n",
                            "    }\n",
                            "</style>\n",
                            "<table border=\"1\" class=\"dataframe\">\n",
                            "  <thead>\n",
                            "    <tr style=\"text-align: right;\">\n",
                            "      <th></th>\n",
                            "      <th>userID</th>\n",
                            "      <th>itemID</th>\n",
                            "      <th>rating</th>\n",
                            "      <th>timestamp</th>\n",
                            "    </tr>\n",
                            "  </thead>\n",
                            "  <tbody>\n",
                            "    <tr>\n",
                            "      <th>0</th>\n",
                            "      <td>196</td>\n",
                            "      <td>242</td>\n",
                            "      <td>3.0</td>\n",
                            "      <td>881250949</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>1</th>\n",
                            "      <td>186</td>\n",
                            "      <td>302</td>\n",
                            "      <td>3.0</td>\n",
                            "      <td>891717742</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>2</th>\n",
                            "      <td>22</td>\n",
                            "      <td>377</td>\n",
                            "      <td>1.0</td>\n",
                            "      <td>878887116</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>3</th>\n",
                            "      <td>244</td>\n",
                            "      <td>51</td>\n",
                            "      <td>2.0</td>\n",
                            "      <td>880606923</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>4</th>\n",
                            "      <td>166</td>\n",
                            "      <td>346</td>\n",
                            "      <td>1.0</td>\n",
                            "      <td>886397596</td>\n",
                            "    </tr>\n",
                            "  </tbody>\n",
                            "</table>\n",
                            "</div>"
                        ],
                        "text/plain": [
                            "  userID itemID  rating  timestamp\n",
                            "0    196    242     3.0  881250949\n",
                            "1    186    302     3.0  891717742\n",
                            "2     22    377     1.0  878887116\n",
                            "3    244     51     2.0  880606923\n",
                            "4    166    346     1.0  886397596"
                        ]
                    },
                    "execution_count": 3,
                    "metadata": {},
                    "output_type": "execute_result"
                }
            ],
            "source": [
                "ratings_df = movielens.load_pandas_df(\n",
                "    size=MOVIELENS_DATA_SIZE,\n",
                "    header=[USER,ITEM,RATING,TIMESTAMP]\n",
                ")\n",
                "\n",
                "# make sure the IDs are loaded as strings to better prevent confusion with embedding ids\n",
                "ratings_df[USER] = ratings_df[USER].astype('str')\n",
                "ratings_df[ITEM] = ratings_df[ITEM].astype('str')\n",
                "\n",
                "ratings_df.head()"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 4,
            "metadata": {},
            "outputs": [],
            "source": [
                "# Split the dataset\n",
                "train_valid_df, test_df = python_stratified_split(\n",
                "    ratings_df, \n",
                "    ratio=0.75, \n",
                "    min_rating=1, \n",
                "    filter_by=\"item\", \n",
                "    col_user=USER, \n",
                "    col_item=ITEM\n",
                ")"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 5,
            "metadata": {},
            "outputs": [],
            "source": [
                "# Remove \"cold\" users from test set  \n",
                "test_df = test_df[test_df.userID.isin(train_valid_df.userID)]"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Training"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 6,
            "metadata": {},
            "outputs": [],
            "source": [
                "# fix random seeds to make sure our runs are reproducible\n",
                "np.random.seed(101)\n",
                "torch.manual_seed(101)\n",
                "torch.cuda.manual_seed_all(101)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 7,
            "metadata": {},
            "outputs": [],
            "source": [
                "with Timer() as preprocess_time:\n",
                "    data = CollabDataLoaders.from_df(train_valid_df, \n",
                "                                     user_name=USER, \n",
                "                                     item_name=ITEM, \n",
                "                                     rating_name=RATING, \n",
                "                                     valid_pct=0)\n"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 8,
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "text/html": [
                            "<table border=\"1\" class=\"dataframe\">\n",
                            "  <thead>\n",
                            "    <tr style=\"text-align: right;\">\n",
                            "      <th></th>\n",
                            "      <th>userID</th>\n",
                            "      <th>itemID</th>\n",
                            "      <th>rating</th>\n",
                            "    </tr>\n",
                            "  </thead>\n",
                            "  <tbody>\n",
                            "    <tr>\n",
                            "      <th>0</th>\n",
                            "      <td>104</td>\n",
                            "      <td>840</td>\n",
                            "      <td>1.0</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>1</th>\n",
                            "      <td>881</td>\n",
                            "      <td>112</td>\n",
                            "      <td>2.0</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>2</th>\n",
                            "      <td>746</td>\n",
                            "      <td>506</td>\n",
                            "      <td>3.0</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>3</th>\n",
                            "      <td>104</td>\n",
                            "      <td>257</td>\n",
                            "      <td>4.0</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>4</th>\n",
                            "      <td>511</td>\n",
                            "      <td>1527</td>\n",
                            "      <td>4.0</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>5</th>\n",
                            "      <td>497</td>\n",
                            "      <td>763</td>\n",
                            "      <td>3.0</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>6</th>\n",
                            "      <td>407</td>\n",
                            "      <td>869</td>\n",
                            "      <td>3.0</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>7</th>\n",
                            "      <td>291</td>\n",
                            "      <td>924</td>\n",
                            "      <td>4.0</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>8</th>\n",
                            "      <td>109</td>\n",
                            "      <td>94</td>\n",
                            "      <td>4.0</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>9</th>\n",
                            "      <td>82</td>\n",
                            "      <td>597</td>\n",
                            "      <td>3.0</td>\n",
                            "    </tr>\n",
                            "  </tbody>\n",
                            "</table>"
                        ],
                        "text/plain": [
                            "<IPython.core.display.HTML object>"
                        ]
                    },
                    "metadata": {},
                    "output_type": "display_data"
                }
            ],
            "source": [
                "data.show_batch()"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "Now we will create a `collab_learner` for the data, which by default uses the [EmbeddingDotBias](https://docs.fast.ai/collab.html#EmbeddingDotBias) model. We will be using 40 latent factors. This will create an embedding for the users and the items that will map each of these to 40 floats as can be seen below. Note that the embedding parameters are not predefined, but are learned by the model.\n",
                "\n",
                "Although ratings can only range from 1-5, we are setting the range of possible ratings to a range from 0 to 5.5 -- that will allow the model to predict values around 1 and 5, which improves accuracy. Lastly, we set a value for weight-decay for regularization."
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 9,
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "text/plain": [
                            "EmbeddingDotBias(\n",
                            "  (u_weight): Embedding(944, 40)\n",
                            "  (i_weight): Embedding(1683, 40)\n",
                            "  (u_bias): Embedding(944, 1)\n",
                            "  (i_bias): Embedding(1683, 1)\n",
                            ")"
                        ]
                    },
                    "execution_count": 9,
                    "metadata": {},
                    "output_type": "execute_result"
                }
            ],
            "source": [
                "learn = collab_learner(data, n_factors=N_FACTORS, y_range=[0,5.5], wd=1e-1)\n",
                "learn.model"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "Now train the model for 5 epochs setting the maximal learning rate. The learner will reduce the learning rate with each epoch using cosine annealing."
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 10,
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "text/html": [
                            "\n",
                            "<style>\n",
                            "    /* Turns off some styling */\n",
                            "    progress {\n",
                            "        /* gets rid of default border in Firefox and Opera. */\n",
                            "        border: none;\n",
                            "        /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
                            "        background-size: auto;\n",
                            "    }\n",
                            "    progress:not([value]), progress:not([value])::-webkit-progress-bar {\n",
                            "        background: repeating-linear-gradient(45deg, #7e7e7e, #7e7e7e 10px, #5c5c5c 10px, #5c5c5c 20px);\n",
                            "    }\n",
                            "    .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
                            "        background: #F44336;\n",
                            "    }\n",
                            "</style>\n"
                        ],
                        "text/plain": [
                            "<IPython.core.display.HTML object>"
                        ]
                    },
                    "metadata": {},
                    "output_type": "display_data"
                },
                {
                    "data": {
                        "text/html": [
                            "<table border=\"1\" class=\"dataframe\">\n",
                            "  <thead>\n",
                            "    <tr style=\"text-align: left;\">\n",
                            "      <th>epoch</th>\n",
                            "      <th>train_loss</th>\n",
                            "      <th>valid_loss</th>\n",
                            "      <th>time</th>\n",
                            "    </tr>\n",
                            "  </thead>\n",
                            "  <tbody>\n",
                            "    <tr>\n",
                            "      <td>0</td>\n",
                            "      <td>0.961789</td>\n",
                            "      <td>None</td>\n",
                            "      <td>00:09</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <td>1</td>\n",
                            "      <td>0.863359</td>\n",
                            "      <td>None</td>\n",
                            "      <td>00:08</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <td>2</td>\n",
                            "      <td>0.750853</td>\n",
                            "      <td>None</td>\n",
                            "      <td>00:07</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <td>3</td>\n",
                            "      <td>0.637868</td>\n",
                            "      <td>None</td>\n",
                            "      <td>00:08</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <td>4</td>\n",
                            "      <td>0.526907</td>\n",
                            "      <td>None</td>\n",
                            "      <td>00:09</td>\n",
                            "    </tr>\n",
                            "  </tbody>\n",
                            "</table>"
                        ],
                        "text/plain": [
                            "<IPython.core.display.HTML object>"
                        ]
                    },
                    "metadata": {},
                    "output_type": "display_data"
                },
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "Took 51.5260 seconds for training.\n"
                    ]
                }
            ],
            "source": [
                "with Timer() as train_time:\n",
                "    learn.fit_one_cycle(EPOCHS, lr_max=5e-3)\n",
                "\n",
                "print(\"Took {} seconds for training.\".format(train_time))"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "Save the learner so it can be loaded back later for inferencing / generating recommendations"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 11,
            "metadata": {},
            "outputs": [],
            "source": [
                "tmp = TemporaryDirectory()\n",
                "model_path = os.path.join(tmp.name, \"movielens_model.pkl\")"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 12,
            "metadata": {},
            "outputs": [],
            "source": [
                "learn.export(model_path)"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "## Generating Recommendations\n",
                "\n",
                "Load the learner from disk."
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 13,
            "metadata": {},
            "outputs": [],
            "source": [
                "learner = load_learner(model_path)"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "Get all users and items that the model knows"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 14,
            "metadata": {},
            "outputs": [],
            "source": [
                "total_users, total_items = learner.dls.classes.values()\n",
                "total_items = total_items[1:]\n",
                "total_users = total_users[1:]"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "Get all users from the test set and remove any users that were know in the training set"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 15,
            "metadata": {},
            "outputs": [],
            "source": [
                "test_users = test_df[USER].unique()\n",
                "test_users = np.intersect1d(test_users, total_users)"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "Build the cartesian product of test set users and all items known to the model"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 16,
            "metadata": {},
            "outputs": [],
            "source": [
                "users_items = cartesian_product(np.array(test_users),np.array(total_items))\n",
                "users_items = pd.DataFrame(users_items, columns=[USER,ITEM])"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {
                "scrolled": false
            },
            "source": [
                "\n",
                "Lastly, remove the user/items combinations that are in the training set -- we don't want to propose a movie that the user has already watched."
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 17,
            "metadata": {
                "scrolled": false
            },
            "outputs": [],
            "source": [
                "training_removed = pd.merge(users_items, train_valid_df.astype(str), on=[USER, ITEM], how='left')\n",
                "training_removed = training_removed[training_removed[RATING].isna()][[USER, ITEM]]"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "### Score the model to find the top K recommendation"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 18,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "Took 5.1570 seconds for 1511060 predictions.\n"
                    ]
                }
            ],
            "source": [
                "with Timer() as test_time:\n",
                "    top_k_scores = score(learner, \n",
                "                         test_df=training_removed,\n",
                "                         user_col=USER, \n",
                "                         item_col=ITEM, \n",
                "                         prediction_col=PREDICTION)\n",
                "\n",
                "print(\"Took {} seconds for {} predictions.\".format(test_time, len(training_removed)))"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "Calculate some metrics for our model"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 19,
            "metadata": {},
            "outputs": [],
            "source": [
                "eval_map = map(test_df, top_k_scores, col_user=USER, col_item=ITEM, \n",
                "               col_rating=RATING, col_prediction=PREDICTION, \n",
                "               relevancy_method=\"top_k\", k=TOP_K)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 20,
            "metadata": {},
            "outputs": [],
            "source": [
                "eval_ndcg = ndcg_at_k(test_df, top_k_scores, col_user=USER, col_item=ITEM, \n",
                "                      col_rating=RATING, col_prediction=PREDICTION, \n",
                "                      relevancy_method=\"top_k\", k=TOP_K)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 21,
            "metadata": {},
            "outputs": [],
            "source": [
                "eval_precision = precision_at_k(test_df, top_k_scores, col_user=USER, col_item=ITEM, \n",
                "                                col_rating=RATING, col_prediction=PREDICTION, \n",
                "                                relevancy_method=\"top_k\", k=TOP_K)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 22,
            "metadata": {},
            "outputs": [],
            "source": [
                "eval_recall = recall_at_k(test_df, top_k_scores, col_user=USER, col_item=ITEM, \n",
                "                          col_rating=RATING, col_prediction=PREDICTION, \n",
                "                          relevancy_method=\"top_k\", k=TOP_K)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 23,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "Model:\t\tLearner\n",
                        "Top K:\t\t10\n",
                        "MAP:\t\t0.024119\n",
                        "NDCG:\t\t0.152808\n",
                        "Precision@K:\t0.139130\n",
                        "Recall@K:\t0.054943\n"
                    ]
                }
            ],
            "source": [
                "print(\"Model:\\t\\t\" + learn.__class__.__name__,\n",
                "      \"Top K:\\t\\t%d\" % TOP_K,\n",
                "      \"MAP:\\t\\t%f\" % eval_map,\n",
                "      \"NDCG:\\t\\t%f\" % eval_ndcg,\n",
                "      \"Precision@K:\\t%f\" % eval_precision,\n",
                "      \"Recall@K:\\t%f\" % eval_recall, sep='\\n')"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "The above numbers are lower than [SAR](../sar_single_node_movielens.ipynb), but expected, since the model is explicitly trying to generalize the users and items to the latent factors. Next look at how well the model predicts how the user would rate the movie. Need to score `test_df` user-items only. "
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 24,
            "metadata": {},
            "outputs": [],
            "source": [
                "scores = score(learner, \n",
                "               test_df=test_df.copy(), \n",
                "               user_col=USER, \n",
                "               item_col=ITEM, \n",
                "               prediction_col=PREDICTION)"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "Now calculate some regression metrics"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 25,
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "Model:\t\t\tLearner\n",
                        "RMSE:\t\t\t0.904589\n",
                        "MAE:\t\t\t0.715827\n",
                        "Explained variance:\t0.356082\n",
                        "R squared:\t\t0.355173\n"
                    ]
                }
            ],
            "source": [
                "eval_r2 = rsquared(test_df, scores, col_user=USER, col_item=ITEM, col_rating=RATING, col_prediction=PREDICTION)\n",
                "eval_rmse = rmse(test_df, scores, col_user=USER, col_item=ITEM, col_rating=RATING, col_prediction=PREDICTION)\n",
                "eval_mae = mae(test_df, scores, col_user=USER, col_item=ITEM, col_rating=RATING, col_prediction=PREDICTION)\n",
                "eval_exp_var = exp_var(test_df, scores, col_user=USER, col_item=ITEM, col_rating=RATING, col_prediction=PREDICTION)\n",
                "\n",
                "print(\"Model:\\t\\t\\t\" + learn.__class__.__name__,\n",
                "      \"RMSE:\\t\\t\\t%f\" % eval_rmse,\n",
                "      \"MAE:\\t\\t\\t%f\" % eval_mae,\n",
                "      \"Explained variance:\\t%f\" % eval_exp_var,\n",
                "      \"R squared:\\t\\t%f\" % eval_r2, sep='\\n')"
            ]
        },
        {
            "cell_type": "markdown",
            "metadata": {},
            "source": [
                "That RMSE is competitive in comparison with other models."
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 26,
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "application/notebook_utils.json+json": {
                            "data": 0.024118782738867094,
                            "encoder": "json",
                            "name": "map"
                        }
                    },
                    "metadata": {
                        "notebook_utils": {
                            "data": true,
                            "display": false,
                            "name": "map"
                        }
                    },
                    "output_type": "display_data"
                },
                {
                    "data": {
                        "application/notebook_utils.json+json": {
                            "data": 0.1528081472533914,
                            "encoder": "json",
                            "name": "ndcg"
                        }
                    },
                    "metadata": {
                        "notebook_utils": {
                            "data": true,
                            "display": false,
                            "name": "ndcg"
                        }
                    },
                    "output_type": "display_data"
                },
                {
                    "data": {
                        "application/notebook_utils.json+json": {
                            "data": 0.13913043478260873,
                            "encoder": "json",
                            "name": "precision"
                        }
                    },
                    "metadata": {
                        "notebook_utils": {
                            "data": true,
                            "display": false,
                            "name": "precision"
                        }
                    },
                    "output_type": "display_data"
                },
                {
                    "data": {
                        "application/notebook_utils.json+json": {
                            "data": 0.05494302697544413,
                            "encoder": "json",
                            "name": "recall"
                        }
                    },
                    "metadata": {
                        "notebook_utils": {
                            "data": true,
                            "display": false,
                            "name": "recall"
                        }
                    },
                    "output_type": "display_data"
                },
                {
                    "data": {
                        "application/notebook_utils.json+json": {
                            "data": 0.9045892929999733,
                            "encoder": "json",
                            "name": "rmse"
                        }
                    },
                    "metadata": {
                        "notebook_utils": {
                            "data": true,
                            "display": false,
                            "name": "rmse"
                        }
                    },
                    "output_type": "display_data"
                },
                {
                    "data": {
                        "application/notebook_utils.json+json": {
                            "data": 0.7158267242352735,
                            "encoder": "json",
                            "name": "mae"
                        }
                    },
                    "metadata": {
                        "notebook_utils": {
                            "data": true,
                            "display": false,
                            "name": "mae"
                        }
                    },
                    "output_type": "display_data"
                },
                {
                    "data": {
                        "application/notebook_utils.json+json": {
                            "data": 0.3560824305444269,
                            "encoder": "json",
                            "name": "exp_var"
                        }
                    },
                    "metadata": {
                        "notebook_utils": {
                            "data": true,
                            "display": false,
                            "name": "exp_var"
                        }
                    },
                    "output_type": "display_data"
                },
                {
                    "data": {
                        "application/notebook_utils.json+json": {
                            "data": 0.35517333876960555,
                            "encoder": "json",
                            "name": "rsquared"
                        }
                    },
                    "metadata": {
                        "notebook_utils": {
                            "data": true,
                            "display": false,
                            "name": "rsquared"
                        }
                    },
                    "output_type": "display_data"
                },
                {
                    "data": {
                        "application/notebook_utils.json+json": {
                            "data": 51.52598460000445,
                            "encoder": "json",
                            "name": "train_time"
                        }
                    },
                    "metadata": {
                        "notebook_utils": {
                            "data": true,
                            "display": false,
                            "name": "train_time"
                        }
                    },
                    "output_type": "display_data"
                },
                {
                    "data": {
                        "application/notebook_utils.json+json": {
                            "data": 5.156951100005244,
                            "encoder": "json",
                            "name": "test_time"
                        }
                    },
                    "metadata": {
                        "notebook_utils": {
                            "data": true,
                            "display": false,
                            "name": "test_time"
                        }
                    },
                    "output_type": "display_data"
                }
            ],
            "source": [
                "# Record results for tests - ignore this cell\n",
                "store_metadata(\"map\", eval_map)\n",
                "store_metadata(\"ndcg\", eval_ndcg)\n",
                "store_metadata(\"precision\", eval_precision)\n",
                "store_metadata(\"recall\", eval_recall)\n",
                "store_metadata(\"rmse\", eval_rmse)\n",
                "store_metadata(\"mae\", eval_mae)\n",
                "store_metadata(\"exp_var\", eval_exp_var)\n",
                "store_metadata(\"rsquared\", eval_r2)\n",
                "store_metadata(\"train_time\", train_time.interval)\n",
                "store_metadata(\"test_time\", test_time.interval)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 27,
            "metadata": {},
            "outputs": [],
            "source": [
                "tmp.cleanup()"
            ]
        },
        {
            "cell_type": "raw",
            "metadata": {},
            "source": []
        }
    ],
    "metadata": {
        "celltoolbar": "Tags",
        "kernelspec": {
            "display_name": "recommenders",
            "language": "python",
            "name": "python3"
        },
        "language_info": {
            "codemirror_mode": {
                "name": "ipython",
                "version": 3
            },
            "file_extension": ".py",
            "mimetype": "text/x-python",
            "name": "python",
            "nbconvert_exporter": "python",
            "pygments_lexer": "ipython3",
            "version": "3.9.16"
        }
    },
    "nbformat": 4,
    "nbformat_minor": 2
}
